print "Updated by Wei 2014/07/09"
print "Please use the allInOne program."
exit(1)
# Updated by Wei on 2014/07/09
# steps to configure this program
# (1) set NUM_OF_TOP_RESULTS_CUTOFF
# (2) set inputFileName2
# (3) set inputFileName1
# (4) set inputFileName0
# (5) set TOTAL_NUM_OF_POSTINGS
# (6) set numOfPostingNeededToBePoppedAtDebugPercentage
# Make the same reports for the clueweb09B, to see the overlap tradeoff
# Updated by Wei on 2014/07/06
# make things work
# Updated by Wei on 2014/06/28
from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
# import matplotlib
import os
# import pylab
import random
import sys
import time
from sets import Set
# from scipy import stats
import numpy as np

def getUnprunedDocumentResultsForQueries(inputFileName1,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,termANDTermIDDict,qIDsWithTopKDocumentResultImpactDict,top10PostingWithDocumentResultDict,qIDsWithTOP10ThresholdDict,top10DocumentResultDict,topKPostingWithDocumentResultDict,NUM_OF_TOP_RESULTS_CUTOFF,qidsToEvaluateDict,qIDsWithTopKDocumentResultNumOfPostingsContributedDict):
    print "getUnprunedDocumentResultsForQueries() function begins..."
    #tempQueryTermDict = {}
    #tempOutputFileName = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
    #tempOutputFileHandler = open(tempOutputFileName,"w")
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfResultsForAllTheQueries = 0
    
    inputFileHandler1 = open(inputFileName1,"r")
    currentLine = inputFileHandler1.readline()
    nextLine = ""
    currentQID = ""
    
    processFlag = False
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = currentLine.strip().split(":")[1].strip()
            print "currentQID:",currentQID
            print "numOfQueriesHavingQID:",numOfQueriesHavingQID
            numOfQueriesHavingQID += 1
            # debug
            #if numOfQueriesHavingQID == 500:
            #    break
            
            if currentQID in qidsToEvaluateDict:
                processFlag = True
            else:
                processFlag = False
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:") and processFlag:
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler1.readline()
            # print nextLine.strip()
            parsedFlag = True
            
            if nextLine.strip().endswith("is NOT in the lexicon."):
                parsedFlag = False
            
            if nextLine.strip() != "" and parsedFlag:
                
                # skip the following sampling line: 
                # [2013-05-24 14:20:44] 38871 terms with their lists length have been loaded.
                # nextLine = inputFileHandler1.readline()
                
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                
                # temp section
                #for queryTermIndex in currentQueryTermIndexDict:
                #    currentQueryTerm = currentQueryTermIndexDict[queryTermIndex]
                #    if currentQueryTerm not in tempQueryTermDict:
                #        tempQueryTermDict[currentQueryTerm] = 1
                #        tempOutputFileHandler.write( currentQueryTerm + "\n")
                
                
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip one line
                currentLine = inputFileHandler1.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler1.readline()
                numOfResultsForTheCurrentQuery = 0
                
                # for IMPORTANT DEBUG ONLY
                #print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                #print "currentLine:",currentLine
                #exit(1)
                
                lineElements = currentLine.strip().split(" ")
                while len( lineElements ) == 25:
                    theRank = int(lineElements[0])
                    theDocID = lineElements[-3]
                    theScore = float(lineElements[-4])
                    documentResultKey = currentQID + "_" + theDocID
                    
                    if theRank <= NUM_OF_TOP_RESULTS_CUTOFF:
                        if currentQID not in qIDsWithTopKDocumentResultImpactDict:
                            qIDsWithTopKDocumentResultImpactDict[currentQID] = {}
                            qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID] = {}
                        if documentResultKey not in qIDsWithTopKDocumentResultImpactDict:
                            qIDsWithTopKDocumentResultImpactDict[currentQID][documentResultKey] = 0.0
                            qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][documentResultKey] = 0
                        else:
                            print "duplicated document result."
                            print "documentResultKey:",documentResultKey
                            exit(1)

                    if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                        qIDsWithTOP10ThresholdDict[currentQID] = theScore;
                        top10DocumentResultDict[documentResultKey] = 1;
                    
                    upperBound = 0
                    if len(currentQueryTermIndexDict) >= 10:
                        upperBound = 10
                    else:
                        upperBound = len(currentQueryTermIndexDict)
                    
                    for i in range(0,upperBound):
                        currentTerm = currentQueryTermIndexDict[i]
                        if currentTerm not in termANDTermIDDict:
                            print currentTerm,"is NOT in the dict."
                            exit(1)
                        else:
                            currentTermID = termANDTermIDDict[currentTerm]
                        currentTermScore = float( lineElements[11 + i] )
                        if currentTermScore != 0.0:
                            postingKey = currentTermID + "_" + theDocID
                            
                            if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                                if postingKey not in top10PostingWithDocumentResultDict:
                                    top10PostingWithDocumentResultDict[postingKey] = []
                                top10PostingWithDocumentResultDict[postingKey].append(documentResultKey)                            
                            
                            if theRank <= NUM_OF_TOP_RESULTS_CUTOFF:
                                if postingKey not in topKPostingWithDocumentResultDict:
                                    topKPostingWithDocumentResultDict[postingKey] = []
                                topKPostingWithDocumentResultDict[postingKey].append(documentResultKey)                    
                    
                    numOfResultsForTheCurrentQuery += 1
                    
                    currentLine = inputFileHandler1.readline()
                    lineElements = currentLine.strip().split(" ")
                    
                numOfResultsForAllTheQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForCurrentQuery:",numOfResultsForTheCurrentQuery
                print
            
        currentLine = inputFileHandler1.readline()
    
    print "numOfQueriesHavingQID:",numOfQueriesHavingQID
    print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
    print "numOfResultsForAllTheQueries:",numOfResultsForAllTheQueries
    print "getUnprunedDocumentResultsForQueries() function ends."
    print

print "Program Begins..."
# for OR settings
QUERY_LENGTH_TO_EVALUATE = 0
print "len(sys.argv):",len(sys.argv)
if len(sys.argv) != 3:
    print "Illegal # of arguments"
    print "Usage: python programName.py all_postings_being_popped_file QUERY_LENGTH_TO_EVALUATE"
    exit(1)
else:
    QUERY_LENGTH_TO_EVALUATE = int(sys.argv[2])

NUM_OF_TOP_RESULTS_CUTOFF = 1000
NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE = 10

# key: qID
# value: original threshold for qID (currently NOT USED)
# in this program, the threshold dict do NOT have any use
qIDsWithTOP10ThresholdDict = {}
# key: qID
# value: a list of topK document results
qIDsWithTopKDocumentResultImpactDict = {}
# key: qID
# value: a list of topK document results
qIDsWithTopKDocumentResultNumOfPostingsContributedDict = {}

# key: topK posting
# value: a list of document results this poting serves
topKPostingWithDocumentResultDict = {}
# key: document results for TOP10
# value: (currently NOT USED)
top10DocumentResultDict = {}
# key: top10 posting
# value: a list of document results this posting serves
top10PostingWithDocumentResultDict = {}


qidsToEvaluateDict = {}
# gov2 tail5K testing queries
inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_tail5K_queries_testing_withQueryLengthAdded"
# clueweb09B testing queries
# inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_queries_ALL_sortedByRandom_testing_withQueryLengthAdded"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentQueryLength = int(lineElements[0])
    if QUERY_LENGTH_TO_EVALUATE == 0:
        currentQID = line.strip().split(":")[0].strip().split(" ")[1]
        qidsToEvaluateDict[currentQID] = 1
    else:      
        if currentQueryLength == QUERY_LENGTH_TO_EVALUATE:
            currentQID = line.strip().split(":")[0].strip().split(" ")[1]
            qidsToEvaluateDict[currentQID] = 1
inputFileHandler.close()
print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)
# print "qidsToEvaluateDict:",qidsToEvaluateDict
# exit(1)

# OR
# for gov2 testing queries
# for pangolin:
# inputFileName1 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/rawResults_50%_TOP10_OR_20140126Night"
# for dodo:
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/rawResults_50%_TOP10_OR_20140126Night"
# for vidaserver1:
inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/rawResults_50%_TOP1000_OR_20140126Night"
# for clueweb09B testing queries
# inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_clueweb09B_TOP1000_testingQueries_OR_20140705"

termANDTermIDDict = {}
# for gov2
# for pangolin:
# inputFileName0 = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
# for dodo:
# inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/partOfLexiconTermsWithTermIDForTail5KQueries"
# for vidaserver1:
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/partOfLexiconTermsWithTermIDForTail5KQueries"
# for clueweb09B
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/partOfLexiconTermsWithTermIDForTestingQueries_clueweb09B"

inputFileHandler = open(inputFileName0,"r")
currentLine = inputFileHandler.readline()
currentLineNum = 0

while currentLine:
    if currentLineNum % 1000000 == 0:
        print currentLineNum,"lines loaded."
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    if currentTerm not in termANDTermIDDict:
        termANDTermIDDict[currentTerm] = currentTermID
    currentLine = inputFileHandler.readline()
    currentLineNum += 1
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
inputFileHandler.close()

getUnprunedDocumentResultsForQueries(inputFileName1,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,termANDTermIDDict,qIDsWithTopKDocumentResultImpactDict,top10PostingWithDocumentResultDict,qIDsWithTOP10ThresholdDict,top10DocumentResultDict,topKPostingWithDocumentResultDict,NUM_OF_TOP_RESULTS_CUTOFF,qidsToEvaluateDict,qIDsWithTopKDocumentResultNumOfPostingsContributedDict)

print "OVERALL:"
print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)
print "QUERY_LENGTH_TO_EVALUATE:",QUERY_LENGTH_TO_EVALUATE
NUM_OF_TOPK_DOCUMENT_RESULT = 0
for currentQID in qIDsWithTopKDocumentResultImpactDict:
    NUM_OF_TOPK_DOCUMENT_RESULT += len( qIDsWithTopKDocumentResultImpactDict[currentQID] )
print "# of topK document results:",NUM_OF_TOPK_DOCUMENT_RESULT
print "len(topKPostingWithDocumentResultDict):",len(topKPostingWithDocumentResultDict)
print "len(top10DocumentResultDict):",len(top10DocumentResultDict)
print "len(top10PostingWithDocumentResultDict):",len(top10PostingWithDocumentResultDict)
print "NUM_OF_TOP_RESULTS_CUTOFF:",NUM_OF_TOP_RESULTS_CUTOFF
print "NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:",NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
#################################################################################################################
# for gov2
TOTAL_NUM_OF_POSTINGS = 6451948010
numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
# clueweb09B
# TOTAL_NUM_OF_POSTINGS = 17075485964
# numOfPostingNeededToBePoppedAtDebugPercentage = 13660388

numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
numOfPostingNeededToBePoppedAt2Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.02)
numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
numOfPostingNeededToBePoppedAt4Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.04)
numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
numOfPostingNeededToBePoppedAt6Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.06)
numOfPostingNeededToBePoppedAt7Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.07)
numOfPostingNeededToBePoppedAt8Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.08)
numOfPostingNeededToBePoppedAt9Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.09)
numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)

print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
print "numOfPostingNeededToBePoppedAt2Percentage:",numOfPostingNeededToBePoppedAt2Percentage
print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
print "numOfPostingNeededToBePoppedAt4Percentage:",numOfPostingNeededToBePoppedAt4Percentage
print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
print "numOfPostingNeededToBePoppedAt6Percentage:",numOfPostingNeededToBePoppedAt6Percentage
print "numOfPostingNeededToBePoppedAt7Percentage:",numOfPostingNeededToBePoppedAt7Percentage
print "numOfPostingNeededToBePoppedAt8Percentage:",numOfPostingNeededToBePoppedAt8Percentage
print "numOfPostingNeededToBePoppedAt9Percentage:",numOfPostingNeededToBePoppedAt9Percentage
print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage

numOfOriginalTOP10PostingsRetainedInPrunedIndex = 0
numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex = 0
numOfBytes = 0
numOfPostingPopped = 0
numOfPostingBeInTOP10 = 0

# options:
# "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_0_WHOLE"
# "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140320Afternoon_weight_1_WHOLE"
# "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140320Afternoon_weight_50_WHOLE"

# the current version
inputFileHandler = open(sys.argv[1],"rb")
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "--->posting file to evaluate:",sys.argv[1]
statinfo = os.stat(sys.argv[1])


fileSize = statinfo.st_size
print "file size:",fileSize
numOfResultsReturnedCountedUpTo10 = 0

while numOfBytes < fileSize:
    # old version
    # each time, just read the info of ONE posting, too few
    # byteString = inputFileHandler.read(4 + 4 + 4)
    # (termID,docID,currentProbability) = unpack( "2I1f", byteString)
    
    # current version
    # each time, read the info of 1M postings
    byteStringBuffer = inputFileHandler.read( 1000000 * 16)
    byteStringBufferIndexPosition = 0
    for i in range(0,1000000):
        byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
        byteStringBufferIndexPosition += 16
        (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
        postingKey = str(termID) + "_" + str(docID)
        
        if postingKey in top10PostingWithDocumentResultDict:
            # debug
            # print "----->",termID,docID
            numOfOriginalTOP10PostingsRetainedInPrunedIndex += 1
            numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex += len(top10PostingWithDocumentResultDict[postingKey])
        # debug
        # print "postingKey:",postingKey
        # print termID,docID,currentProbability,impactScore
        if postingKey in topKPostingWithDocumentResultDict:
            # interesting issue solved
            # print "touched."
            for currentDocumentResultKey in topKPostingWithDocumentResultDict[postingKey]:
                currentQID = currentDocumentResultKey.strip().split("_")[0]
                qIDsWithTopKDocumentResultImpactDict[currentQID][currentDocumentResultKey] += impactScore
                qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][currentDocumentResultKey] += 1
                # tempCounter += 1

        if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt2Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt4Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt6Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt7Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt8Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt9Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
            
            numOfOriginalTOP10DocumentResultRetainedAtTOP10Position = 0
            numOfOriginalTOP10PostingRetainedAtTOP10Position = 0
            currentQIDList = []
            currentQIDList = qIDsWithTopKDocumentResultImpactDict.keys()
            currentQIDList.sort(cmp=None, key=None, reverse=False)
            
            for currentQID in currentQIDList:
                currentQIDDocumentResultList = []
                for currentDocumentResultKey in qIDsWithTopKDocumentResultImpactDict[currentQID]:
                    currentDocumentResultImpactScore = qIDsWithTopKDocumentResultImpactDict[currentQID][currentDocumentResultKey]
                    if currentDocumentResultImpactScore == 0.0:
                        pass
                    else:
                        currentQIDDocumentResultList.append( (currentDocumentResultKey,currentDocumentResultImpactScore) )
                
                currentQIDDocumentResultList.sort(cmp=None, key=itemgetter(1), reverse=True)
                # debug
                #if currentQID == "95074":
                #    print currentQID
                #    print len(currentQIDDocumentResultList)
                #    print len(currentQIDDocumentResultList[:10])
                #    print currentQIDDocumentResultList
                #    print currentQIDDocumentResultList[:10]
                #    print
                #    exit(1)
                for currentDocumentResultTuple in currentQIDDocumentResultList[:10]:
                    (thisDocumentResult,impactScore) = currentDocumentResultTuple
                    if thisDocumentResult in top10DocumentResultDict:
                        numOfOriginalTOP10DocumentResultRetainedAtTOP10Position += 1
                        numOfOriginalTOP10PostingRetainedAtTOP10Position += qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][thisDocumentResult]
                        thisDocID = thisDocumentResult.strip().split("_")[1]
                        print currentQID,thisDocID,impactScore,"NYU_IRTK"
            
            print "OVERALL:"
            print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)
            print "QUERY_LENGTH_TO_EVALUATE:",QUERY_LENGTH_TO_EVALUATE
            print "NUM_OF_TOPK_DOCUMENT_RESULT:",NUM_OF_TOPK_DOCUMENT_RESULT
            print "len(topKPostingWithDocumentResultDict):",len(topKPostingWithDocumentResultDict)
            print "len(top10DocumentResultDict):",len(top10DocumentResultDict)
            print "len(top10PostingWithDocumentResultDict):",len(top10PostingWithDocumentResultDict)
            print "NUM_OF_TOP_RESULTS_CUTOFF:",NUM_OF_TOP_RESULTS_CUTOFF
            print "NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:",NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
            print "numOfOriginalTOP10DocumentResultRetainedAtTOP10Position:",numOfOriginalTOP10DocumentResultRetainedAtTOP10Position
            print "numOfOriginalTOP10PostingRetainedAtTOP10Position:",numOfOriginalTOP10PostingRetainedAtTOP10Position
            print "numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex:",numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex
            print "numOfOriginalTOP10PostingsRetainedInPrunedIndex:",numOfOriginalTOP10PostingsRetainedInPrunedIndex
            print "numOfPostingPopped:",numOfPostingPopped
        numOfBytes += 12
        numOfPostingPopped += 1
        if numOfPostingPopped % 1000000 == 0:
            print str(numOfPostingPopped),"postings have been examined."

inputFileHandler.close()
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
exit(1)
